{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "00f1d67d-fa17-4417-8ba9-57c83342b6c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os.path as osp\n",
    "from lxml import etree\n",
    "from selenium import webdriver\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import shutil\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "71aa8cd0-6430-48f7-b172-313defabdd4d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = r'D:\\Hresource\\SpiderPic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f0929b22-af15-4821-a61f-92b0114ef560",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_path = r\"D:\\Hresource\\PickedSpiderPics\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "62277a97-e06d-404c-88bb-15312fbc9ab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = r\"D:\\Hresource\\Models\\res34.pth\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "66a5d125-852b-4e53-abbf-a76aa5a9e66f",
   "metadata": {},
   "outputs": [],
   "source": [
    "broser= webdriver.Edge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e9bfe66a-8121-4b10-a253-9a2ea75652d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_raw_url = 'https://www.fulitu.cc/page/{}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "efa921c2-5967-4904-8970-85174ec29700",
   "metadata": {},
   "outputs": [],
   "source": [
    "for page in range(1, 75):\n",
    "    req = requests.get(target_raw_url.format(page))\n",
    "    html = etree.HTML((req.text))\n",
    "    href = html.xpath('//a[@class=\"item-link\"]/@href')\n",
    "    for i in href:\n",
    "        broser.get(i)\n",
    "        img_list = [url.get_attribute('src') for url in broser.find_elements('xpath', '//img[@class=\"post-item-img lazy\"]')]\n",
    "        if len(img_list) > 0:\n",
    "            for imgs in img_list:\n",
    "                name = str(int(time.time() * 1000))\n",
    "                name = osp.join(path, name + '.jpg')\n",
    "                with open(name, 'wb') as fp:\n",
    "                    fp.write(requests.get(imgs).content)\n",
    "                    fp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "55384992-8f71-41a4-aae8-f4803059cc2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "trans = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Resize([224,224])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8d4f9dd2-2329-4eb8-9dab-7a75f067be0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "imgs = [osp.join(path, f) for f in os.listdir(path)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a41d9352-908b-4abf-bc48-e08eeea1c10d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torchvision.models.resnet34(pretrained=True)\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Sequential(\n",
    "    nn.Linear(num_ftrs, 1),\n",
    "    nn.Sigmoid()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "30e2b36c-7c13-4f20-a371-395f72b0d338",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(model_path))\n",
    "model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8fe03d3e-1add-47eb-a890-7acec6f39912",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in imgs:\n",
    "    try:\n",
    "        img = trans(plt.imread(i)).unsqueeze(0).cuda()\n",
    "    except:\n",
    "        continue\n",
    "    tar = 1 if model(img) > 0.1 else 0\n",
    "    if tar == 1:\n",
    "        shutil.move(i, osp.join(new_path, osp.basename(i)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
